/**
 * 
 */
package ca.uwindsor.cs.deepweb.utility.wikipedia;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.util.Vector;

/**
 * @author Liang Jie
 *
 */
public class EnWikiToFilesGenerator {
	/**
	 * Instance of buffered reader to read the data source file
	 */
	protected BufferedReader bufferedreader_in;

	/**
	 * The content of a title
	 */
	protected String string_content;

	/**
	 * The title in Wikipedia
	 */
	protected String string_title;
	
	protected long offset;
	
	protected boolean ignoresmallfiles;
	
	private char[] chars;
	
	private StringBuffer temp;
	
	public EnWikiToFilesGenerator(String wikifile){
		offset = 0;
		InputStreamReader read = null;
		try {
			read = new InputStreamReader(new FileInputStream(
					wikifile),"UTF-8");
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} 
		ignoresmallfiles = false;
		bufferedreader_in = new BufferedReader(read);
	}
	
	protected void filter() {
		chars = string_content.toCharArray();
		temp = new StringBuffer();
		for (int offset_1 = 0; offset_1 < chars.length; offset_1++) {
			if (chars[offset_1] == '<') {
				for (int offset_2 = offset_1 + 1; offset_2 < chars.length; offset_2++) {
					if (chars[offset_2] == '>') {
						offset_1 = offset_2;
						break;
					}
				}
			} else {
				temp.append(chars[offset_1]);
			}
		}

		string_content = temp.toString().trim();
	}
	
	public void generate(String path) {
		String string_line = new String();
		Vector<String> v = new Vector<String>();
		try {
			string_line = bufferedreader_in.readLine();
			StringBuffer bu;
			FileOutputStream fp;
			StringBuffer strbuf;
			int i=0;
			while (string_line != null) {
				i++;
				if (string_line.startsWith("</>")) {
					if (v.size() > 2) {
						string_title = v.get(0).trim();
						string_content = v.get(1) + "\n";
						bu = new StringBuffer();
						for (int offset = 2; offset < v.size(); offset++) {
							bu.append(v.get(offset));
							bu.append("\n");
						}
						string_content += bu.toString();
					} else {
						string_title = v.get(0).trim();
						string_content = v.get(1).trim();
					}
					filter();
					
					if(ignoresmallfiles){
						if(string_content.length()<1024){
							v.clear();
							string_line = bufferedreader_in.readLine();
							continue;
						}
					}
					
					//create a file and save the title and content to it
					strbuf= new StringBuffer();
					strbuf.append(path);
					strbuf.append(java.io.File.separator);
					strbuf.append("zhwiki_");
					strbuf.append(offset);
					strbuf.append(".txt");
					fp = new FileOutputStream(strbuf.toString());
					string_title += "\n";
					fp.write(string_title.getBytes("UTF-8"));
					fp.write(string_content.getBytes("UTF-8"));
					fp.flush();
					fp.close();
					System.out.println("extracted: "+strbuf);
					offset++;

					v.clear();
					string_line = bufferedreader_in.readLine();
				} else if(string_line.startsWith("&gt;")){
					v.clear();
					bufferedreader_in.readLine();
					string_line = bufferedreader_in.readLine();
				}else {
					v.add(string_line);
					string_line = bufferedreader_in.readLine();
				}
				if(i%1000==0){
					System.gc();
				}
			}
		} catch (IOException e1) {
			// TODO Auto-generated catch block
			e1.printStackTrace();
		}
		System.out.println("Generation complete!");
		System.out.println("Total extracted: "+offset);
	}
	
//	protected String processFilename(String title){
//		String filename = new String();
//		filename = title.replaceAll("\\", "_");
//		return filename;
//	}
	
	/**
	 * @param args
	 */
	public static void main(String[] args) {
//		args = new String[2];
//		args[0] = "D:\\content.txt";
//		args[1] = "D:\\temp";
		if(args.length!=2){
			System.exit(-1);
		}
		EnWikiToFilesGenerator gen = new EnWikiToFilesGenerator(args[0]);
		gen.generate(args[1]);

	}
}
